package com.yjjxt

import org.apache.spark._
import org.apache.spark.rdd._

object Hello06CheckPoint {
  def main(args: Array[String]): Unit = {
    val sparkConf = new SparkConf().setMaster("local").setAppName("Hello06CheckPoint" + System.currentTimeMillis())
    val sparkContext = new SparkContext(sparkConf)
    sparkContext.setCheckpointDir("./checkpoint" + System.currentTimeMillis() % 10000000)
    val lines: RDD[String] = sparkContext.textFile("src/main/resources/NASA_access_log_Aug95")
    val words: RDD[String] = lines.flatMap(_.split(" "))
    println("words" + words.getNumPartitions)
    words.cache
    words.checkpoint
    words.count
    sparkContext.stop
  }
}
